setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df2 = data.frame(lapply(df, mean_impute))
v = cor(df2[, c(-780)], df2[,780])
u = as.vector(v)
names(u) = unlist(dimnames(v)[1])
u = u[!is.na(u)]
u[rev(order(abs(u)))][1:10]0

#      f471       f536        f12        f11       f674 
#0.03984884 0.02565173 0.02004074 0.02004074 0.01942627 

quant.f471 = quantile(df2$f471, seq(0, 1, .2))
grp.f471 = cut(df2$f471, quant.f471,  1:5,include.lowest=T)

df3 = df2[-1,]

sort(sapply(df3[,2:780],function(x) mean(df3$loss[x >= quantile(x, .999)]>0)), decr=T)[2:15]

sort(sapply(df3[,2:780],function(x) mean(df3$loss[x <= quantile(x, .001)]>0)), decr=T)[1:10]


ind1 = df3$f471>3.5
ind2 = df3$f468>1.8
ind3 = df3$f471>7
ind4 = df3$f468>4

require(quantreg)
m = rq(df3$loss~ind1 + ind2 + ind1*ind2 + ind3 + ind4)
m$coef
mean(abs(m$resid))

 0.7987011

pred = data.frame(id=df3$id, pred=predict(m))
df_flag = merge(df3, flag_id, by="id")
df_flag2 = merge(df_flag, pred, by="id")

u=data.frame(loss=df_flag$loss, loss_ind=df_flag$loss>0, flag=df_flag$flag, pred=predict(m, newdata=df_flag))







#df_test = read.csv("test.csv")
df_test = readRDS("~/accept_decline/test.rds")
pred_test = predict(m, newdata=df_test, type="response")
keep_ind = pred_test>=.08

df2 = data.frame(df2, loss_ind=df2$loss>0)
m=glm(formula = loss_ind ~ f527 + f528, family = binomial(), data = df2)



df3=df2[pred>=.08,]




ind4_test = !is.na(df_test$f468) & df_test$f468 > 4
ind1_test = !is.na(df_test$f471) & df_test$f471>3.5
ind2_test = !is.na(df_test$f468) & df_test$f468>1.8
ind3_test = !is.na(df_test$f471) & df_test$f471>7



pred_test = ind1_test*3 + ind2_test*3 + ind3_test*1 + ind1_test*ind2_test*(-1) +ind4_test*(-1)
pred_test=pred_test*keep_ind
submission = data.frame(id=df_test$id, loss = pred_test)
test_ids = read.csv("test_ids.csv", stringsAsFactors=F)
valid_ind = submission$id %in% test_ids$id
submission$loss = submi

write.csv(submission, "submission.csv", row.names=F)



x=merge(submission, test_ids, by="id")

rel = function(i)
{
	u = tapply(df2[,i], df2$loss_ind, median)
	v = u[2]/u[1]
	names(v) = names(df2)[i]
	return (v)
}

sapply(1:ncol(df2), rel)
sort(sapply(1:ncol(df2), rel), decr=T)[1:20]

    f471      f468      f229      f230      f249      f250      f479      f260 
0.7289720 0.6981132 0.6836735 0.6509434 0.6371681 0.6363636 0.6330275 0.6132075 
     f566      f746      f259      f269      f536      f270 
0.6090909 0.6037736 0.6000000 0.5929204 0.5925926 0.5877193 


> quantile(df3$f471, .995)
99.5% 
 1.64 
 
 > quantile(df3$f468, .995)
 99.5% 
1.1686 

> quantile(df3$f229, .995)
99.5% 
 1.76 

> quantile(df3$f230, .995)
 99.5% 
2.3586 

99.5% 
 1.64 



